#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @File  : SinaSpider.py
# @Author: tongnian
# @Date  : 2018/11/22
# @Desc  :

import requests, threading
from bs4 import BeautifulSoup
import time
from SpiderAction.News import *
from selenium import webdriver

# https请求头
header = {
    'User-Agent': ' Mozilla/5.0 (Windows NT 6.1; Win64; x64)'
                  ' AppleWebKit/537.36 (KHTML, like Gecko)'
                  ' Chrome/64.0.3282.186 Safari/537.36',
    'Connection': 'keep-alive'
}


def getSinaArticleInfo(news_url):

    web_data = requests.get(news_url)
    web_data.encoding = 'utf-8'

    soup = BeautifulSoup(web_data.text,'lxml')
    # 标题
    title = soup.select('.main-title')[0].text

    # 日期
    time = soup.select('.date')[0].contents[0].strip()
    # dt = datetime.strptime(time,'%Y年%m月%d日%H:%M')
    # print(dt)

    # 引用源
    source = soup.select('.date-source a')[0].text
    source_href = soup.select('.date-source a')[0].get('href')

    # 正文
    article = str(soup.select('.article'))



    sinaNews = NewsSpider(title=title, date=time, sourceName=source, sourceUrl=source_href,
                          article=article, status="wait",spider_source="Sina_Chnia",
                          examine="suspend")

    sinaNews.save()


def getSinaNewsList():
    news_url = 'https://news.sina.com.cn/china/'
    driver = webdriver.Firefox()
    driver.get(news_url)
    time.sleep(5)  # 象征性地等一下
    newsList = driver.find_elements_by_xpath("//div[@class='feed-card-item']")
    num = len(newsList)
    print(num)
    for i in range(num):
        Feed = driver.find_elements_by_xpath("//div[@class='feed-card-item']/h2/a")[i].text
        html_url = driver.find_elements_by_xpath("//div[@class='feed-card-item']/h2/a")[i].get_attribute("href")
        news_time = driver.find_elements_by_xpath("//div[@class='feed-card-time']")[i].text
        pstime = '分钟前'
        if pstime in news_time:
            getSinaArticleInfo(html_url)
            print("保存新闻")
    print("新闻爬取结束")
    driver.close()


def get_sinanews():
    getSinaNewsList()
    timer = threading.Timer(3600, get_sinanews)  # 一小时调用一次函数
    timer.start()


if __name__ == '__main__':
    get_sinanews()



